{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "\"\"\"\n",
    "# @file name  : train_lenet.py\n",
    "# @author     : tingsongyu\n",
    "# @date       : 2019-09-07 10:08:00\n",
    "# @brief      : 人民币分类模型训练\n",
    "\"\"\"\n",
    "\"\"\"\n",
    "# @file name  : train_lenet_gpu.py\n",
    "# @modified by: greebear\n",
    "# @date       : 2019-10-26 13:25:00\n",
    "# @brief      : 猫狗分类模型训练\n",
    "\"\"\"\n",
    "import os\n",
    "import random\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import DataLoader\n",
    "import torchvision.transforms as transforms\n",
    "import torch.optim as optim\n",
    "from matplotlib import pyplot as plt\n",
    "from lenet import LeNet, MyNet\n",
    "from DogCat_dataset import DogCatDataset\n",
    "\n",
    "\n",
    "def set_seed(seed=1):\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)\n",
    "\n",
    "\n",
    "set_seed()  # 设置随机种子\n",
    "\n",
    "# 参数设置\n",
    "MAX_EPOCH = 40\n",
    "BATCH_SIZE = 512\n",
    "LR = 0.001\n",
    "log_interval = 10\n",
    "val_interval = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================ step 1/5 数据 ============================\n",
    "\n",
    "split_dir = os.path.join(\"..\", \"data\", \"cad_split\")\n",
    "train_dir = os.path.join(split_dir, \"train\")\n",
    "valid_dir = os.path.join(split_dir, \"valid\")\n",
    "\n",
    "norm_mean = [0.485, 0.456, 0.406]\n",
    "norm_std = [0.229, 0.224, 0.225]\n",
    "\n",
    "train_transform = transforms.Compose([\n",
    "    transforms.Resize((224, 224)),\n",
    "    transforms.RandomCrop(224, padding=4),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(norm_mean, norm_std),\n",
    "])\n",
    "\n",
    "valid_transform = transforms.Compose([\n",
    "    transforms.Resize((224, 224)),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(norm_mean, norm_std),\n",
    "])\n",
    "\n",
    "# 构建MyDataset实例\n",
    "train_data = DogCatDataset(data_dir=train_dir, transform=train_transform)\n",
    "valid_data = DogCatDataset(data_dir=valid_dir, transform=valid_transform)\n",
    "\n",
    "# 构建DataLoder\n",
    "train_loader = DataLoader(dataset=train_data, batch_size=BATCH_SIZE, shuffle=True, num_workers=16)\n",
    "valid_loader = DataLoader(dataset=valid_data, batch_size=BATCH_SIZE, num_workers=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================ step 2/5 模型 ============================\n",
    "\n",
    "net = MyNet(classes=2)\n",
    "net.initialize_weights()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MyNet(\n",
       "  (conv1): Conv2d(3, 16, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3))\n",
       "  (bn1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (conv2_1): Conv2d(16, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
       "  (bn2_1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (conv2_2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "  (bn2_2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (conv3_1): Conv2d(32, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
       "  (bn3_1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (conv3_2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "  (bn3_2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (conv4_1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
       "  (bn4_1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (conv4_2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "  (bn4_2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (avg_pool): AdaptiveAvgPool2d(output_size=(1, 1))\n",
       "  (bn5_avg): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (max_pool): AdaptiveMaxPool2d(output_size=(1, 1))\n",
       "  (bn5_max): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (dropout1): Dropout(p=0.25)\n",
       "  (fc1): Linear(in_features=256, out_features=128, bias=True)\n",
       "  (dropout2): Dropout(p=0.5)\n",
       "  (fc2): Linear(in_features=128, out_features=2, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net.to(\"cuda\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================ step 3/5 损失函数 ============================\n",
    "criterion = nn.CrossEntropyLoss()   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================ step 4/5 优化器 ============================\n",
    "optimizer = optim.Adam(net.parameters(), lr=LR, betas=(0.9, 0.999), eps=1e-08, weight_decay=1e-1)                        # 选择优化器\n",
    "scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.1)     # 设置学习率下降策略"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# inputs.to(\"cuda\").device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "            Conv2d-1         [-1, 16, 112, 112]           2,368\n",
      "       BatchNorm2d-2         [-1, 16, 112, 112]              32\n",
      "            Conv2d-3           [-1, 32, 56, 56]           4,640\n",
      "       BatchNorm2d-4           [-1, 32, 56, 56]              64\n",
      "            Conv2d-5           [-1, 32, 56, 56]           9,248\n",
      "       BatchNorm2d-6           [-1, 32, 56, 56]              64\n",
      "            Conv2d-7           [-1, 64, 28, 28]          18,496\n",
      "       BatchNorm2d-8           [-1, 64, 28, 28]             128\n",
      "            Conv2d-9           [-1, 64, 28, 28]          36,928\n",
      "      BatchNorm2d-10           [-1, 64, 28, 28]             128\n",
      "           Conv2d-11          [-1, 128, 14, 14]          73,856\n",
      "      BatchNorm2d-12          [-1, 128, 14, 14]             256\n",
      "           Conv2d-13          [-1, 128, 14, 14]         147,584\n",
      "      BatchNorm2d-14          [-1, 128, 14, 14]             256\n",
      "AdaptiveAvgPool2d-15            [-1, 128, 1, 1]               0\n",
      "      BatchNorm2d-16            [-1, 128, 1, 1]             256\n",
      "AdaptiveMaxPool2d-17            [-1, 128, 1, 1]               0\n",
      "      BatchNorm2d-18            [-1, 128, 1, 1]             256\n",
      "          Dropout-19                  [-1, 256]               0\n",
      "           Linear-20                  [-1, 128]          32,896\n",
      "          Dropout-21                  [-1, 128]               0\n",
      "           Linear-22                    [-1, 2]             258\n",
      "================================================================\n",
      "Total params: 327,714\n",
      "Trainable params: 327,714\n",
      "Non-trainable params: 0\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.14\n",
      "Forward/backward pass size (MB): 8.43\n",
      "Params size (MB): 1.25\n",
      "Estimated Total Size (MB): 9.82\n",
      "----------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "from torchsummary import summary\n",
    "summary(net, input_size=(3, 112, 112))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_acc = list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training:Epoch[000/040] Iteration[010/040] Loss: 1.1257 Acc:53.55%\n",
      "Training:Epoch[000/040] Iteration[020/040] Loss: 0.9067 Acc:54.85%\n",
      "Training:Epoch[000/040] Iteration[030/040] Loss: 0.8051 Acc:56.00%\n",
      "Training:Epoch[000/040] Iteration[040/040] Loss: 0.7202 Acc:56.97%\n",
      "Valid:\t Epoch[000/040] Iteration[005/005] Loss: 3.2791 【Acc:58.28%】\n",
      "Training:Epoch[001/040] Iteration[010/040] Loss: 0.7098 Acc:60.10%\n",
      "Training:Epoch[001/040] Iteration[020/040] Loss: 0.6851 Acc:60.38%\n",
      "Training:Epoch[001/040] Iteration[030/040] Loss: 0.6392 Acc:61.52%\n",
      "Training:Epoch[001/040] Iteration[040/040] Loss: 0.6527 Acc:61.80%\n",
      "Valid:\t Epoch[001/040] Iteration[005/005] Loss: 3.1807 【Acc:63.84%】\n",
      "Training:Epoch[002/040] Iteration[010/040] Loss: 0.6567 Acc:63.28%\n",
      "Training:Epoch[002/040] Iteration[020/040] Loss: 0.6268 Acc:63.86%\n",
      "Training:Epoch[002/040] Iteration[030/040] Loss: 0.6064 Acc:64.96%\n",
      "Training:Epoch[002/040] Iteration[040/040] Loss: 0.5873 Acc:66.03%\n",
      "Valid:\t Epoch[002/040] Iteration[005/005] Loss: 3.9561 【Acc:54.12%】\n",
      "Training:Epoch[003/040] Iteration[010/040] Loss: 0.5944 Acc:68.54%\n",
      "Training:Epoch[003/040] Iteration[020/040] Loss: 0.5745 Acc:69.45%\n",
      "Training:Epoch[003/040] Iteration[030/040] Loss: 0.5489 Acc:70.38%\n",
      "Training:Epoch[003/040] Iteration[040/040] Loss: 0.5573 Acc:70.86%\n",
      "Valid:\t Epoch[003/040] Iteration[005/005] Loss: 3.0032 【Acc:68.08%】\n",
      "Training:Epoch[004/040] Iteration[010/040] Loss: 0.5748 Acc:70.61%\n",
      "Training:Epoch[004/040] Iteration[020/040] Loss: 0.5583 Acc:71.38%\n",
      "Training:Epoch[004/040] Iteration[030/040] Loss: 0.5339 Acc:72.25%\n",
      "Training:Epoch[004/040] Iteration[040/040] Loss: 0.5235 Acc:72.84%\n",
      "Valid:\t Epoch[004/040] Iteration[005/005] Loss: 2.8907 【Acc:70.56%】\n",
      "Training:Epoch[005/040] Iteration[010/040] Loss: 0.5269 Acc:74.38%\n",
      "Training:Epoch[005/040] Iteration[020/040] Loss: 0.5171 Acc:74.75%\n",
      "Training:Epoch[005/040] Iteration[030/040] Loss: 0.5065 Acc:75.01%\n",
      "Training:Epoch[005/040] Iteration[040/040] Loss: 0.4878 Acc:75.35%\n",
      "Valid:\t Epoch[005/040] Iteration[005/005] Loss: 2.8069 【Acc:71.20%】\n",
      "Training:Epoch[006/040] Iteration[010/040] Loss: 0.5290 Acc:74.20%\n",
      "Training:Epoch[006/040] Iteration[020/040] Loss: 0.5245 Acc:74.36%\n",
      "Training:Epoch[006/040] Iteration[030/040] Loss: 0.4954 Acc:75.38%\n",
      "Training:Epoch[006/040] Iteration[040/040] Loss: 0.4784 Acc:75.83%\n",
      "Valid:\t Epoch[006/040] Iteration[005/005] Loss: 3.7415 【Acc:57.08%】\n",
      "Training:Epoch[007/040] Iteration[010/040] Loss: 0.5154 Acc:75.49%\n",
      "Training:Epoch[007/040] Iteration[020/040] Loss: 0.4872 Acc:76.46%\n",
      "Training:Epoch[007/040] Iteration[030/040] Loss: 0.4667 Acc:77.14%\n",
      "Training:Epoch[007/040] Iteration[040/040] Loss: 0.4648 Acc:77.62%\n",
      "Valid:\t Epoch[007/040] Iteration[005/005] Loss: 2.7953 【Acc:70.96%】\n",
      "Training:Epoch[008/040] Iteration[010/040] Loss: 0.5380 Acc:73.09%\n",
      "Training:Epoch[008/040] Iteration[020/040] Loss: 0.5060 Acc:74.83%\n",
      "Training:Epoch[008/040] Iteration[030/040] Loss: 0.4990 Acc:75.51%\n",
      "Training:Epoch[008/040] Iteration[040/040] Loss: 0.5015 Acc:75.88%\n",
      "Valid:\t Epoch[008/040] Iteration[005/005] Loss: 2.8695 【Acc:69.00%】\n",
      "Training:Epoch[009/040] Iteration[010/040] Loss: 0.5304 Acc:74.65%\n",
      "Training:Epoch[009/040] Iteration[020/040] Loss: 0.4910 Acc:76.15%\n",
      "Training:Epoch[009/040] Iteration[030/040] Loss: 0.4598 Acc:77.15%\n",
      "Training:Epoch[009/040] Iteration[040/040] Loss: 0.4424 Acc:77.81%\n",
      "Valid:\t Epoch[009/040] Iteration[005/005] Loss: 2.5077 【Acc:75.40%】\n",
      "Training:Epoch[010/040] Iteration[010/040] Loss: 0.4472 Acc:79.22%\n",
      "Training:Epoch[010/040] Iteration[020/040] Loss: 0.4079 Acc:80.85%\n",
      "Training:Epoch[010/040] Iteration[030/040] Loss: 0.4009 Acc:81.65%\n",
      "Training:Epoch[010/040] Iteration[040/040] Loss: 0.3753 Acc:82.25%\n",
      "Valid:\t Epoch[010/040] Iteration[005/005] Loss: 2.1286 【Acc:79.68%】\n",
      "Training:Epoch[011/040] Iteration[010/040] Loss: 0.3844 Acc:84.14%\n",
      "Training:Epoch[011/040] Iteration[020/040] Loss: 0.3716 Acc:84.47%\n",
      "Training:Epoch[011/040] Iteration[030/040] Loss: 0.3583 Acc:84.61%\n",
      "Training:Epoch[011/040] Iteration[040/040] Loss: 0.3563 Acc:84.80%\n",
      "Valid:\t Epoch[011/040] Iteration[005/005] Loss: 1.9326 【Acc:83.96%】\n",
      "Training:Epoch[012/040] Iteration[010/040] Loss: 0.3522 Acc:85.92%\n",
      "Training:Epoch[012/040] Iteration[020/040] Loss: 0.3538 Acc:85.75%\n",
      "Training:Epoch[012/040] Iteration[030/040] Loss: 0.3485 Acc:85.83%\n",
      "Training:Epoch[012/040] Iteration[040/040] Loss: 0.3413 Acc:85.89%\n",
      "Valid:\t Epoch[012/040] Iteration[005/005] Loss: 1.9378 【Acc:82.24%】\n",
      "Training:Epoch[013/040] Iteration[010/040] Loss: 0.3467 Acc:86.37%\n",
      "Training:Epoch[013/040] Iteration[020/040] Loss: 0.3312 Acc:86.38%\n",
      "Training:Epoch[013/040] Iteration[030/040] Loss: 0.3263 Acc:86.76%\n",
      "Training:Epoch[013/040] Iteration[040/040] Loss: 0.3397 Acc:86.84%\n",
      "Valid:\t Epoch[013/040] Iteration[005/005] Loss: 2.5177 【Acc:72.28%】\n",
      "Training:Epoch[014/040] Iteration[010/040] Loss: 0.3371 Acc:86.54%\n",
      "Training:Epoch[014/040] Iteration[020/040] Loss: 0.3122 Acc:87.10%\n",
      "Training:Epoch[014/040] Iteration[030/040] Loss: 0.3074 Acc:87.45%\n",
      "Training:Epoch[014/040] Iteration[040/040] Loss: 0.2913 Acc:87.76%\n",
      "Valid:\t Epoch[014/040] Iteration[005/005] Loss: 1.5420 【Acc:86.92%】\n",
      "Training:Epoch[015/040] Iteration[010/040] Loss: 0.2975 Acc:88.40%\n",
      "Training:Epoch[015/040] Iteration[020/040] Loss: 0.2917 Acc:88.86%\n",
      "Training:Epoch[015/040] Iteration[030/040] Loss: 0.2863 Acc:88.76%\n",
      "Training:Epoch[015/040] Iteration[040/040] Loss: 0.2989 Acc:88.85%\n",
      "Valid:\t Epoch[015/040] Iteration[005/005] Loss: 1.5113 【Acc:88.48%】\n",
      "Training:Epoch[016/040] Iteration[010/040] Loss: 0.3117 Acc:87.50%\n",
      "Training:Epoch[016/040] Iteration[020/040] Loss: 0.2900 Acc:88.07%\n",
      "Training:Epoch[016/040] Iteration[030/040] Loss: 0.2849 Acc:88.37%\n",
      "Training:Epoch[016/040] Iteration[040/040] Loss: 0.2787 Acc:88.78%\n",
      "Valid:\t Epoch[016/040] Iteration[005/005] Loss: 1.8539 【Acc:84.32%】\n",
      "Training:Epoch[017/040] Iteration[010/040] Loss: 0.3039 Acc:88.12%\n",
      "Training:Epoch[017/040] Iteration[020/040] Loss: 0.2829 Acc:88.65%\n",
      "Training:Epoch[017/040] Iteration[030/040] Loss: 0.2717 Acc:89.03%\n",
      "Training:Epoch[017/040] Iteration[040/040] Loss: 0.2454 Acc:89.50%\n",
      "Valid:\t Epoch[017/040] Iteration[005/005] Loss: 1.5633 【Acc:86.72%】\n",
      "Training:Epoch[018/040] Iteration[010/040] Loss: 0.2570 Acc:90.57%\n",
      "Training:Epoch[018/040] Iteration[020/040] Loss: 0.2580 Acc:90.52%\n",
      "Training:Epoch[018/040] Iteration[030/040] Loss: 0.2407 Acc:90.82%\n",
      "Training:Epoch[018/040] Iteration[040/040] Loss: 0.2421 Acc:90.98%\n",
      "Valid:\t Epoch[018/040] Iteration[005/005] Loss: 1.7774 【Acc:83.40%】\n",
      "Training:Epoch[019/040] Iteration[010/040] Loss: 0.2760 Acc:89.47%\n",
      "Training:Epoch[019/040] Iteration[020/040] Loss: 0.2493 Acc:90.23%\n",
      "Training:Epoch[019/040] Iteration[030/040] Loss: 0.2480 Acc:90.24%\n",
      "Training:Epoch[019/040] Iteration[040/040] Loss: 0.2372 Acc:90.62%\n",
      "Valid:\t Epoch[019/040] Iteration[005/005] Loss: 1.9740 【Acc:80.08%】\n",
      "Training:Epoch[020/040] Iteration[010/040] Loss: 0.2269 Acc:92.27%\n",
      "Training:Epoch[020/040] Iteration[020/040] Loss: 0.2133 Acc:92.58%\n",
      "Training:Epoch[020/040] Iteration[030/040] Loss: 0.2056 Acc:92.77%\n",
      "Training:Epoch[020/040] Iteration[040/040] Loss: 0.2057 Acc:92.91%\n",
      "Valid:\t Epoch[020/040] Iteration[005/005] Loss: 1.1763 【Acc:90.72%】\n",
      "Training:Epoch[021/040] Iteration[010/040] Loss: 0.2082 Acc:93.07%\n",
      "Training:Epoch[021/040] Iteration[020/040] Loss: 0.1973 Acc:93.37%\n",
      "Training:Epoch[021/040] Iteration[030/040] Loss: 0.1984 Acc:93.34%\n",
      "Training:Epoch[021/040] Iteration[040/040] Loss: 0.2016 Acc:93.41%\n",
      "Valid:\t Epoch[021/040] Iteration[005/005] Loss: 1.1379 【Acc:91.48%】\n",
      "Training:Epoch[022/040] Iteration[010/040] Loss: 0.1893 Acc:93.91%\n",
      "Training:Epoch[022/040] Iteration[020/040] Loss: 0.2026 Acc:93.52%\n",
      "Training:Epoch[022/040] Iteration[030/040] Loss: 0.1950 Acc:93.61%\n",
      "Training:Epoch[022/040] Iteration[040/040] Loss: 0.2057 Acc:93.50%\n",
      "Valid:\t Epoch[022/040] Iteration[005/005] Loss: 1.1173 【Acc:91.00%】\n",
      "Training:Epoch[023/040] Iteration[010/040] Loss: 0.1891 Acc:93.59%\n",
      "Training:Epoch[023/040] Iteration[020/040] Loss: 0.1931 Acc:93.68%\n",
      "Training:Epoch[023/040] Iteration[030/040] Loss: 0.1882 Acc:93.77%\n",
      "Training:Epoch[023/040] Iteration[040/040] Loss: 0.1869 Acc:93.77%\n",
      "Valid:\t Epoch[023/040] Iteration[005/005] Loss: 1.1157 【Acc:91.28%】\n",
      "Training:Epoch[024/040] Iteration[010/040] Loss: 0.1894 Acc:94.28%\n",
      "Training:Epoch[024/040] Iteration[020/040] Loss: 0.1886 Acc:94.07%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training:Epoch[024/040] Iteration[030/040] Loss: 0.1858 Acc:94.11%\n",
      "Training:Epoch[024/040] Iteration[040/040] Loss: 0.1954 Acc:94.11%\n",
      "Valid:\t Epoch[024/040] Iteration[005/005] Loss: 1.0870 【Acc:91.64%】\n",
      "Training:Epoch[025/040] Iteration[010/040] Loss: 0.1871 Acc:93.98%\n",
      "Training:Epoch[025/040] Iteration[020/040] Loss: 0.1898 Acc:93.87%\n",
      "Training:Epoch[025/040] Iteration[030/040] Loss: 0.1765 Acc:94.20%\n",
      "Training:Epoch[025/040] Iteration[040/040] Loss: 0.2005 Acc:94.27%\n",
      "Valid:\t Epoch[025/040] Iteration[005/005] Loss: 1.0762 【Acc:91.48%】\n",
      "Training:Epoch[026/040] Iteration[010/040] Loss: 0.1860 Acc:93.69%\n",
      "Training:Epoch[026/040] Iteration[020/040] Loss: 0.1844 Acc:93.77%\n",
      "Training:Epoch[026/040] Iteration[030/040] Loss: 0.1864 Acc:93.80%\n",
      "Training:Epoch[026/040] Iteration[040/040] Loss: 0.1752 Acc:93.90%\n",
      "Valid:\t Epoch[026/040] Iteration[005/005] Loss: 1.0784 【Acc:91.72%】\n",
      "Training:Epoch[027/040] Iteration[010/040] Loss: 0.1807 Acc:94.57%\n",
      "Training:Epoch[027/040] Iteration[020/040] Loss: 0.1714 Acc:94.65%\n",
      "Training:Epoch[027/040] Iteration[030/040] Loss: 0.1772 Acc:94.66%\n",
      "Training:Epoch[027/040] Iteration[040/040] Loss: 0.1750 Acc:94.53%\n",
      "Valid:\t Epoch[027/040] Iteration[005/005] Loss: 1.0677 【Acc:91.84%】\n",
      "Training:Epoch[028/040] Iteration[010/040] Loss: 0.1765 Acc:94.65%\n",
      "Training:Epoch[028/040] Iteration[020/040] Loss: 0.1725 Acc:94.55%\n",
      "Training:Epoch[028/040] Iteration[030/040] Loss: 0.1728 Acc:94.53%\n",
      "Training:Epoch[028/040] Iteration[040/040] Loss: 0.1661 Acc:94.70%\n",
      "Valid:\t Epoch[028/040] Iteration[005/005] Loss: 1.0405 【Acc:92.04%】\n",
      "Training:Epoch[029/040] Iteration[010/040] Loss: 0.1753 Acc:94.63%\n",
      "Training:Epoch[029/040] Iteration[020/040] Loss: 0.1649 Acc:94.89%\n",
      "Training:Epoch[029/040] Iteration[030/040] Loss: 0.1684 Acc:94.88%\n",
      "Training:Epoch[029/040] Iteration[040/040] Loss: 0.1613 Acc:94.92%\n",
      "Valid:\t Epoch[029/040] Iteration[005/005] Loss: 1.0438 【Acc:92.12%】\n",
      "Training:Epoch[030/040] Iteration[010/040] Loss: 0.1725 Acc:94.86%\n",
      "Training:Epoch[030/040] Iteration[020/040] Loss: 0.1569 Acc:95.14%\n",
      "Training:Epoch[030/040] Iteration[030/040] Loss: 0.1622 Acc:95.17%\n",
      "Training:Epoch[030/040] Iteration[040/040] Loss: 0.1773 Acc:95.15%\n",
      "Valid:\t Epoch[030/040] Iteration[005/005] Loss: 1.0221 【Acc:92.20%】\n",
      "Training:Epoch[031/040] Iteration[010/040] Loss: 0.1664 Acc:95.10%\n",
      "Training:Epoch[031/040] Iteration[020/040] Loss: 0.1595 Acc:95.32%\n",
      "Training:Epoch[031/040] Iteration[030/040] Loss: 0.1603 Acc:95.35%\n",
      "Training:Epoch[031/040] Iteration[040/040] Loss: 0.1684 Acc:95.21%\n",
      "Valid:\t Epoch[031/040] Iteration[005/005] Loss: 1.0116 【Acc:92.40%】\n",
      "Training:Epoch[032/040] Iteration[010/040] Loss: 0.1660 Acc:95.08%\n",
      "Training:Epoch[032/040] Iteration[020/040] Loss: 0.1590 Acc:95.33%\n",
      "Training:Epoch[032/040] Iteration[030/040] Loss: 0.1662 Acc:95.13%\n",
      "Training:Epoch[032/040] Iteration[040/040] Loss: 0.1616 Acc:95.12%\n",
      "Valid:\t Epoch[032/040] Iteration[005/005] Loss: 1.0178 【Acc:92.16%】\n",
      "Training:Epoch[033/040] Iteration[010/040] Loss: 0.1599 Acc:95.51%\n",
      "Training:Epoch[033/040] Iteration[020/040] Loss: 0.1625 Acc:95.38%\n",
      "Training:Epoch[033/040] Iteration[030/040] Loss: 0.1655 Acc:95.35%\n",
      "Training:Epoch[033/040] Iteration[040/040] Loss: 0.1649 Acc:95.34%\n",
      "Valid:\t Epoch[033/040] Iteration[005/005] Loss: 1.0114 【Acc:92.16%】\n",
      "Training:Epoch[034/040] Iteration[010/040] Loss: 0.1621 Acc:95.18%\n",
      "Training:Epoch[034/040] Iteration[020/040] Loss: 0.1626 Acc:95.32%\n",
      "Training:Epoch[034/040] Iteration[030/040] Loss: 0.1596 Acc:95.36%\n",
      "Training:Epoch[034/040] Iteration[040/040] Loss: 0.1619 Acc:95.28%\n",
      "Valid:\t Epoch[034/040] Iteration[005/005] Loss: 1.0132 【Acc:92.32%】\n",
      "Training:Epoch[035/040] Iteration[010/040] Loss: 0.1635 Acc:95.04%\n",
      "Training:Epoch[035/040] Iteration[020/040] Loss: 0.1622 Acc:95.35%\n",
      "Training:Epoch[035/040] Iteration[030/040] Loss: 0.1577 Acc:95.44%\n",
      "Training:Epoch[035/040] Iteration[040/040] Loss: 0.1706 Acc:95.45%\n",
      "Valid:\t Epoch[035/040] Iteration[005/005] Loss: 1.0092 【Acc:92.28%】\n",
      "Training:Epoch[036/040] Iteration[010/040] Loss: 0.1592 Acc:95.49%\n",
      "Training:Epoch[036/040] Iteration[020/040] Loss: 0.1615 Acc:95.38%\n",
      "Training:Epoch[036/040] Iteration[030/040] Loss: 0.1567 Acc:95.53%\n",
      "Training:Epoch[036/040] Iteration[040/040] Loss: 0.1694 Acc:95.39%\n",
      "Valid:\t Epoch[036/040] Iteration[005/005] Loss: 1.0115 【Acc:92.24%】\n",
      "Training:Epoch[037/040] Iteration[010/040] Loss: 0.1611 Acc:95.23%\n",
      "Training:Epoch[037/040] Iteration[020/040] Loss: 0.1616 Acc:95.13%\n",
      "Training:Epoch[037/040] Iteration[030/040] Loss: 0.1596 Acc:95.26%\n",
      "Training:Epoch[037/040] Iteration[040/040] Loss: 0.1606 Acc:95.20%\n",
      "Valid:\t Epoch[037/040] Iteration[005/005] Loss: 1.0052 【Acc:92.52%】\n",
      "Training:Epoch[038/040] Iteration[010/040] Loss: 0.1612 Acc:95.23%\n",
      "Training:Epoch[038/040] Iteration[020/040] Loss: 0.1606 Acc:95.36%\n",
      "Training:Epoch[038/040] Iteration[030/040] Loss: 0.1634 Acc:95.35%\n",
      "Training:Epoch[038/040] Iteration[040/040] Loss: 0.1596 Acc:95.45%\n",
      "Valid:\t Epoch[038/040] Iteration[005/005] Loss: 1.0080 【Acc:92.40%】\n",
      "Training:Epoch[039/040] Iteration[010/040] Loss: 0.1592 Acc:95.23%\n",
      "Training:Epoch[039/040] Iteration[020/040] Loss: 0.1577 Acc:95.41%\n",
      "Training:Epoch[039/040] Iteration[030/040] Loss: 0.1600 Acc:95.43%\n",
      "Training:Epoch[039/040] Iteration[040/040] Loss: 0.1672 Acc:95.36%\n",
      "Valid:\t Epoch[039/040] Iteration[005/005] Loss: 1.0023 【Acc:92.60%】\n"
     ]
    }
   ],
   "source": [
    "# ============================ step 5/5 训练 ============================\n",
    "train_curve = list()\n",
    "valid_curve = list()\n",
    "\n",
    "for epoch in range(MAX_EPOCH):\n",
    "\n",
    "    loss_mean = 0.\n",
    "    correct = 0.\n",
    "    total = 0.\n",
    "\n",
    "    net.train()\n",
    "    for i, data in enumerate(train_loader):\n",
    "        # forward\n",
    "        inputs, labels = data\n",
    "        inputs = inputs.to(\"cuda\")\n",
    "        labels = labels.to(\"cuda\")\n",
    "        outputs = net(inputs)\n",
    "\n",
    "        # backward\n",
    "        optimizer.zero_grad()\n",
    "        loss = criterion(outputs, labels)\n",
    "        loss.backward()\n",
    "\n",
    "        # update weights\n",
    "        optimizer.step()\n",
    "\n",
    "        # 统计分类情况\n",
    "        _, predicted = torch.max(outputs.data, 1)\n",
    "        total += labels.size(0)\n",
    "        correct += (predicted == labels).squeeze().sum().cpu().numpy()\n",
    "\n",
    "        # 打印训练信息\n",
    "        loss_mean += loss.item()\n",
    "        train_curve.append(loss.item())\n",
    "        if (i+1) % log_interval == 0:\n",
    "            loss_mean = loss_mean / log_interval\n",
    "            print(\"Training:Epoch[{:0>3}/{:0>3}] Iteration[{:0>3}/{:0>3}] Loss: {:.4f} Acc:{:.2%}\".format(\n",
    "                epoch, MAX_EPOCH, i+1, len(train_loader), loss_mean, correct / total))\n",
    "            loss_mean = 0.\n",
    "\n",
    "    scheduler.step()  # 更新学习率\n",
    "\n",
    "    # validate the model\n",
    "    if (epoch+1) % val_interval == 0:\n",
    "\n",
    "        correct_val = 0.\n",
    "        total_val = 0.\n",
    "        loss_val = 0.\n",
    "        net.eval()\n",
    "        with torch.no_grad():\n",
    "            for j, data in enumerate(valid_loader):\n",
    "                inputs, labels = data\n",
    "                inputs = inputs.to(\"cuda\")\n",
    "                labels = labels.to(\"cuda\")\n",
    "                outputs = net(inputs)\n",
    "                loss = criterion(outputs, labels)\n",
    "\n",
    "                _, predicted = torch.max(outputs.data, 1)\n",
    "                total_val += labels.size(0)\n",
    "                correct_val += (predicted == labels).squeeze().sum().cpu().numpy()\n",
    "\n",
    "                loss_val += loss.item()\n",
    "\n",
    "            valid_curve.append(loss.item())\n",
    "            valid_acc.append(correct_val / total_val)\n",
    "            print(\"Valid:\\t Epoch[{:0>3}/{:0>3}] Iteration[{:0>3}/{:0>3}] Loss: {:.4f} 【Acc:{:.2%}】\".format(\n",
    "                epoch, MAX_EPOCH, j+1, len(valid_loader), loss_val, correct_val / total_val))\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdd3zU9f3A8df7LouEBEjCXoGwkSEEZMlQlqhgLVpxi9ZWrfZXtYqzrlarHWq1VutuVVTcIHWDixVk7xUgrISElZCQ3N3n98f37nKXXJJLyCU57v18PHjk7js/Oc33fZ/1/ogxBqWUUpHL1tAFUEop1bA0ECilVITTQKCUUhFOA4FSSkU4DQRKKRXhohq6ADWVmppq0tLSGroYSikVVpYvX37QGNMy0L6wCwRpaWlkZmY2dDGUUiqsiMjOyvZp05BSSkU4DQRKKRXhNBAopVSEC7s+AqWUqonS0lKys7MpLi5u6KLUi7i4ODp06EB0dHTQ52ggUEqd0rKzs0lMTCQtLQ0RaejihJQxhry8PLKzs+nSpUvQ52nTkFLqlFZcXExKSsopHwQARISUlJQa1340ECilTnmREAQ8avO7Rkwg2LT/GH/7fBMHC040dFGUUqpRiZhAsDWngKe/3kp+YUlDF0UpFSHy8vIYOHAgAwcOpE2bNrRv3977vqQkuGfRNddcw6ZNm0JazojpLLa5a0suXYhHKVVPUlJSWLlyJQAPPPAATZs25fbbb/c7xhiDMQabLfD38ldeeSXk5YyYGoGn3czlauCCKKUi3tatW+nTpw+XXXYZffv2Zd++fVx//fVkZGTQt29fHnroIe+xo0aNYuXKlTgcDpo3b86sWbMYMGAAw4cPJycnp07KozUCpVTEePCTdazfe7ROr9mnXRJ/OL9vjc/buHEjr7/+OhkZGQA89thjJCcn43A4GDduHNOnT6dPnz5+5xw5coQxY8bw2GOPceutt/Lyyy8za9ask/4dIqZGYHPXCDQOKKUag/T0dG8QAHjrrbcYNGgQgwYNYsOGDaxfv77COU2aNOGcc84BYPDgwWRlZdVJWSKnRuAOeVojUCpy1eabe6gkJCR4X2/ZsoWnnnqKpUuX0rx5cy6//PKAcwFiYmK8r+12Ow6Ho07KEjE1Am8fgQYCpVQjc/ToURITE0lKSmLfvn189tln9Xr/yKkReANBAxdEKaXKGTRoEH369KFXr1507tyZkSNH1uv9xYToG7KIvAycB+QYY06r4rghwCLgEmPMnOqum5GRYWqzMM13W3K54qWlvPvr4QxJS67x+Uqp8LRhwwZ69+7d0MWoV4F+ZxFZbozJCHR8KJuGXgUmV3WAiNiBPwOfh7AcANi9w0e1SqCUUr5CFgiMMd8C+dUcdjPwHlA3g2GrINo0pJRSATVYZ7GItAd+BjxXH/fzzCMIVVOYUkqFq4YcNfQkcKcxptq5viJyvYhkikhmbm5urW5ms2mNQCmlAmnIUUMZwGx3k00qMEVEHMaYD8sfaIx5AXgBrM7i2txMZxYrpVRgDRYIjDHe5XNE5FVgbqAgUFd0HoFSSgUWsqYhEXkLa1hoTxHJFpFrReTXIvLrUN2zKppiQinVEMaNG1dhgtiTTz7JDTfcUOk5TZs2BWDv3r1Mnz494DFjx46lNkPpAwlZjcAYM6MGx14dqnJ4aNOQUqohzJgxg9mzZzNp0iTvttmzZ/P4449Xe267du2YM6fa6VUnLWJSTHhqBE7tLVZK1aPp06czb94870I0WVlZ7N27l9NPP52zzz6bQYMG0a9fPz766KMK52ZlZXHaadZ83KKiIi655BJ69+7Nz372M4qKiuqsjJpiQikVOebPgv1r6vaabfrBOY9Vujs5OZmhQ4cyf/58pk2bxuzZs7n44otp0qQJH3zwAUlJSRw8eJBhw4YxderUStccfu6554iPj2fDhg2sXr2aQYMG1dmvEDk1AvdvqvMIlFL1zdM8BFaz0IwZMzDGcPfdd9O/f3/Gjx/Pnj17OHDgQKXX+Pbbb7n88ssB6N+/P/3796+z8mmNQCkVOar45h5K06ZN43e/+x0//fQTx48fZ/Dgwbz66qvk5uayfPlyoqOjSUtLC5h6uj5ETo1AO4uVUg2kadOmjBs3jpkzZzJjhjWO5siRI7Rq1Yro6Gi++eYbdu7cWeU1Ro8ezZtvvgnA2rVrWb16dZ2VL2ICgc4jUEo1pBkzZrBq1SpvILjsssvIzMykX79+vP766/Tq1avK82+44QYKCgro3bs3999/P4MHD66zskVc05DGAaVUQ7jgggv8+ihTU1NZtGhRwGMLCgoASEtLY+3atYC1TKWnn6GuRUyNQJuGlFIqsAgKBNpZrJRSgURMIPAMzdWFaZSKPJE0bLw2v2vEBAK7LYSdxYeyYNXbdX9dpdRJi4uLIy8vLyKCgTGGvLw84uLianRexHUWh6RCkPky/PAU9L0AomJDcAOlVG116NCB7OxsaruWSbiJi4ujQ4cONTonYgKBhLKzuMC90ubxfEhqW/fXV0rVWnR0NF26dKn+wAgWMU1DZcNHQxkIDtb9tZVSKsQiLhCEpGmo0B0ICjUQKKXCTwQFAutnaJqG3G2Px/Pq/tpKKRViERMIJFQ1ApcLCjUQKKXCV8QEAk+NoM77CIoOgXFar7VpSCkVhiIoEIRohTJP/wBoZ7FSKiyFcvH6l0UkR0TWVrL/MhFZLSJrRORHERkQqrKA74SyOr5wgU8g0BqBUioMhbJG8CowuYr9O4Axxph+wMPACyEsS+jmEXj6B+JTrXkESikVZkI2ocwY862IpFWx/0eft4uBmk2Fq6GQzSPw1Aha9S4LCkopFUYaSx/BtcD8UN4gZPMICnPAFg0p6do0pJQKSw2eYkJExmEFglFVHHM9cD1Ap06danWfkM0jKMiFhJbWv6J8aziprbHEV6WUql6DPrFEpD/wIjDNGFPpIHxjzAvGmAxjTEbLli1rey8AnvxyS63Or1RhDjRtafURGBcUH67b6yulVIg1WCAQkU7A+8AVxpjNDVWOk1aQAwmtID7Feq/NQ0qpMBOypiEReQsYC6SKSDbwByAawBjzL+B+IAX4p/vbusMYkxGq8oRMYS607gsJ7kBw/CDQo0GLpJRSNRHKUUMzqtl/HXBdqO5fL4yxAkGCu2kItEaglAo72qt5MooPg7MEmraCBHcg0HxDSqkwo4HgZHiyjvr2EWiaCaVUmNFAcDI8eYaatrSWqIxJhEKtESilwosGgpPhmVWc4B7SmpCiNQKlVNjRQHAyCn2ahsCdb0hrBEqp8KKB4GQU5IDYID7Zeh+foqOGlFJhRwPBySjMsWoBNrv1PkFrBEqp8KOB4GQU5FpDRz08NYJQrIuslFIhEpGB4LstdZQuujCnrKMYrBqB8wSUFNbN9ZVSqh5EZCC44qWldXOhQDUC0JFDSqmwEpGBoE4YU7FG4E0zof0ESqnwoYGgtk4cA0exf43Am2ZCawRKqfARUYHglrO7193Fys8hgLJhpDpySCkVRiIqEDSJttfdxQp80kt4aAZSpVQYiqhA4Fmusk548gz51ghiE8Eeo01DSqmwEmGBoA4jgbdG4BMIRKxagXYWK6XCSEQFgrqMA1YfgZQ1B3nEp2gfgVIqrERYIKjjGkF8MtjLLfKmGUiVUmEmogJB3fYR5Pr3D3jEp2pnsVIqrERUIPCNA/mFJSd3sYIc/xFDHpp4TikVZkIWCETkZRHJEZG1lewXEXlaRLaKyGoRGRSqsng4fXLBuU42MVxhTiU1ghQ4cRQcJxlolFKqnoSyRvAqMLmK/ecA3d3/rgeeC2FZAHC5yh7+TtdJBoLyeYY8vPmGtFaglAoPIQsExphvgfwqDpkGvG4si4HmItI2VOUBcJo6CgQlhVBa6J9nyEPTTCilwkxD9hG0B3b7vM92b6tARK4XkUwRyczNrX0KaWdd1Qg86SUC1gh0drFSKryERWexMeYFY0yGMSajZcsA38KDVGdNQwUB8gx5aNOQUirMNGQg2AN09Hnfwb0tZHybhhwnVSMIkGfIw9s0pIFAKRUeGjIQfAxc6R49NAw4YozZF8ob+qaYeOWHHbW/UEGAPEMeTVoAok1DSqmwEcrho28Bi4CeIpItIteKyK9F5NfuQz4FtgNbgX8DN4aqLB4xUWW/7htLdlFU4qzdhbwpqAPUCGx2a8axdhYrpcJEVPWH1I4xZkY1+w1wU6juH0iM3T/ulThcNImpRWrqghyIaw5RMYH3a74hpVQYCYvO4rriWyMAKHbUtkaQE3jEkIdmIFVKhZGICgTn9fefpnCi1FW7CxVUkmfIQxPPKaXCSFCBQEQ6i8h49+smIpIY2mKFRvP4GP577Rne93e8t4pSZy2CQWEleYY8NPGcUiqMVBsIROSXwBzgefemDsCHoSxUKNl9UpAu3p7PVxtyan6R6moE8SlQlA+uWtY4lFKqHgVTI7gJGAkcBTDGbAGqeAo2buU7h2ucmrq0GE4cqbpGkJAKxgXFh2teQKWUqmfBBIITxhhvKk0RiQJOMmNbw0lLifd7b69pJCisYlaxh6aZUEqFkWACwUIRuRtoIiITgHeBT0JbrNBpHu8/5LPG6xgXBliruLwET5oJDQRKqcYvmEAwC8gF1gC/wpoIdm8oC1WfbDWtEVSVZ8hD8w0ppcJItYHAGOMyxvzbGHORMWa6+3XYNg0B3Htub+9re61rBNWMGgJtGlJKhYVqZxaLyA4C9AkYY7qGpET1YGDH5t7XNe4srirPkIeuSaCUCiPBpJjI8HkdB1wEJIemOPXDtzmoxk1DhbkQmwTRcZUfExULMYk6u1gpFRaCaRrK8/m3xxjzJHBuPZQtZGrcQeyrICdwsrny4pO1j0ApFRaCaRryXVTehlVDCFmyuvrg2y/gqum6BIWVrFVcXkKqNg0ppcJCMA/0v/q8dgBZwMUhKU09sfnUg2568ydW3D8x+JMLcqBlz+qPi0+FYyFdXkEppepEtYHAGDOuPgpSn3wnkR06XlqzkwtzoMuZ1R+XkAoH1tawZEopVf8qDQQicmtVJxpj/lb3xakf5fsIjpc4iI8JonLkLIWiQ1WPGPLw9BEYAyfTJ6GUUiFWVWdxYjX/wlb5QPDUl1uCO9GTXqKqOQQe8angKIaSwhqWTiml6lelX4ONMQ/WZ0HqU/kRoyccQWYJDWYOgYfvXILYpsEXTiml6lkwo4bigGuBvljzCAAwxswMYblCylVuYnT5lcsq5a0RBNM05JldnAct0oIvnFJK1bNgnoD/AdoAk4CFWOsRHAvm4iIyWUQ2ichWEZkVYH8nEflGRFaIyGoRmVKTwteWo9yQ0fJrGVfKWyMIpmlI8w0ppcJDME/AbsaY+4BCY8xrWJPJzqjmHETEDjwLnAP0AWaISJ9yh90LvGOMOR24BPhnTQpfW87ygSDoGkEQmUc9NAOpUipMBPME9IyvPCwipwHNCG5hmqHAVmPMdvd6BrOBaeWOMUCS+3UzYG8Q1z1p5VNRBx0ICnIhOgFiEqo/VhPPKaXCRDBPwBdEpAVwH/AxsB74cxDntQd2+7zPdm/z9QBwuYhkY6W3vjnQhUTkehHJFJHM3NzcIG5dTcGaN+EvFw3wvs/MOhTcidWtVewrNhHsMVojUEo1esEEgleMMYeMMQuNMV2NMa2MMc9Xf1pQZgCvGmM6AFOA/4hIhTIZY14wxmQYYzJatgzyQVyNs3qVVWq+3HCA3GMnqj+pICe4EUNgzR2IT9E+AqVUoxdMINghIi+IyNkiNZoZtQfo6PO+g3ubr2uBdwCMMYuwRiWl1uAetZYQ6792cfl+g4CCzTPkEZ+qGUiVUo1eMIGgF/Al1iL2WSLyjIiMCuK8ZUB3EekiIjFYncEflztmF3A2gIj0xgoEJ9/2E4TYKP9A8OWGA9WfFGzmUY+EFG0aUko1esGkoT5ujHnHGHMhMBCrc3dhEOc5gN8AnwEbsEYHrRORh0Rkqvuw24Bfisgq4C3g6oZa/ezeD6vJC+R0WM08Na4RaCBQSjVuQQ2XEZExIvJPYDnWt/agso8aYz41xvQwxqQbY/7o3na/MeZj9+v1xpiRxpgBxpiBxpjPa/l71MrcmytWbP7+xWY27Dta8eDjeYCpWY1A+wiUUmGg2kAgIlnA/wHfAf2MMRcbY94LdcHqQ5tm/quMnXA4eeqrLUx/7seKB9dkDoFHQiqcOAqOkpMopVJKhVYw6xH0N8YE+Ioc/sonn5v56jIASp0BWqdqkmfIw3d2cVLb2hRRKaVCLpg+glMyCEDF5HM/bLWacQwBAkFN8gx5xOvsYqVU4xfklNpTU2UL11ddI6jJqCGdXayUavwiOxDUZFpEYQ5ExVkzhoPlSTOhHcZKqUYsmM7i34pIklheEpGfRKQGi/w2XpVUCAIryLX6B2oSPBI0ECilGr9gagQz3f0EE4EWwBXAYyEtVT2pSY0gLyebDcdiySsIIhWFR5MWgGjTkFKqUQsmEHiellOA/xhj1vlsC2s1CQQH9u4muzSRh+eur8EN7FYw0M5ipVQjFkwgWC4in2MFgs9EJBEIcm3Hxq2qpqG0WfNYuNkaKXTPB2tIlSMcNM3IOXaCz9btD/4mCanaNKSUatSCCQTXArOAIcaY40A0cE1IS1VP7NV0Ejz79VYA3lySRTJHOUgzftyWx6/+s5z8wiAniWniOaVUIxdMIBgObDLGHBaRy7FWFTsS2mLVDxFh25/KVsc8f0A7v/1Ls/J56JP1JHOMKHFx0DTz7nM4g6wUxSdr05BSqlELJhA8BxwXkQFYSeK2Aa+HtFT1yLdW8I8Zp7Px4cl++1/+YQd9bDsB2Gb8A0UwTHwqhYcOsDv/+MkVVCmlQiSYQOBwZwSdBjxjjHkWqMFg+vASF20n2u7fZDTEthGHsbHC1c27LdgUqfkkEVd6mF+9vqwOS6mUUnUnmEBwTETuwho2Os+9glh0aIvVsKYO8F9Rc4hsZr3pTCFNvNuCWsgGcDVJxi6GGMcp0ZqmlDoFBRMIfgGcwJpPsB9rpbEnQlqqejb35lG8cd0Z3vePXtjP+zoaBwNtW8l09fQ7Z+P+4FIwmSZWvqEk1ymbskkpFeaCSTq3H3gDaCYi5wHFxphTpo8A4LT2zRjZrWyFzJioso+lr2TRREpY6urld87MVzODurbLnXguyXm4DkqqlFJ1L5gUExcDS4GLsBakWSIi00NdsMZiiG0jAMtdPao9ttTpwlWuyci48w0lao1AKdVIBdM0dA/WHIKrjDFXAkOB+0JbrMZjiG0TO1ytyaV5hX0XP7+I3729EoDrX8+k+z3zufcj/yUvHXHJADQzgfsIVmcf5g8fraWBVuhUSqmgAoHNGJPj8z4vyPNOAYYM2yYyXT3plBxfYe/SHfl8sGIPAJ+vPwDAm0t2+R3jiG0BQDNX4EDwi+cX89qinRSVOuuy4EopFbRgHuj/E5HPRORqEbkamAd8GszFRWSyiGwSka0iMquSYy4WkfUisk5E3gy+6KGV3jKBdNlLshTQd/hkPrl5FK/PHBrw2PLNQb6c9lgKTBzNTt31fZRSYa7apSqNMb8XkZ8DI92bXjDGfFDdeSJiB54FJgDZwDIR+dgYs97nmO7AXcBIY8whEanB8l+hNf+3o5n7yg+wB/qcMQmaRDO6R+BFabre7R8Xi0qcNImxA+AysMV0YLBZF/Bcz2po2jKklGooQTXxGGPeM8bc6v5XbRBwGwpsNcZsN8aUALOxJqX5+iXwrDHmkPs+OTQSMVE2Lkzdba1IlpJeo3N3+cwidroMHzpH0pMs2L+2wrGeAOAMUSR4c8kuco/VIHW2UiriVBoIROSYiBwN8O+YiATTztEe2O3zPtu9zVcPoIeI/CAii0VkMgGIyPUikikimbm5uUHcuo7s/BE6DfNbjObFKzP49ZiqA8POvELva5cxfOIcTqmxU7LirUrPMSHI57o15xh3f7CG295dVfcXV0qdMioNBMaYRGNMUoB/icaYpDq6fxTQHRgLzAD+LSIVhucYY14wxmQYYzJatqzBmsEn4+heOLwTOg332zy+T2tmndOLzikVO489rv/Pco4UlQLgcllpJha4BnBs2VtkbvcPZJ56gCsENYL8QqsMx0846vzaSqlTRyhH/+wBOvq87+De5isb+NgYU2qM2QFsxgoMDW/XYutnp2EBd1+c0THgdg9PkjnPA/5955mkuPJ48sWXAh4fiqahQncAaBpXbVdQ5MjZCMXaca+Ur1AGgmVAdxHpIiIxwCXAx+WO+RCrNoCIpGI1FW0PYZmCt2sRRCdAmwEBd984Np3bJlQ+yewPH1udw54H/Neu0zlq4rnQ/h2rs31mGbuf/6GoERxzB4KE2MCB4J3M3fz9i811ft9Gq+Q4vDAWfniqoUuiVKMSskBgjHEAvwE+AzYA7xhj1onIQyIy1X3YZ0CeiKwHvgF+b4xpHKu47FoEHTLAHvghKiLcNM7KRto5JZ4tfzyHq0ekefcv33mIEoeL5VmHADhBDHOdw5hsW8Ylz3xV4XquSvoIDhacIG3WPN5Ztjvg/jMf/5r7P6rYCQ1wrNhqGkqsJBDcMWc1T321JfCNT0V7fwJHEeRsaOiSKNWohLTNwBjzKeXmHBhj7vd5bYBb3f8aj+IjcGAdjL6jysNsNiHrsXO97/9wfh9e/THL+77HvfP9jn/fOYpLo75msm0pt7/bjWVZ+d59LmNwugw2sYKMx/Zcq+P5obnrOb1Tc7q39s8Avju/iNcX7eT2ST1JivNPCnus2KoReIayRrzdS6yf+dsathxKNTIRMkO4hrKXWcN4KukfqIzvAzyQTNOTXa6W/Mz+PXOWZ7Mzr2yY6W9nryD97k/54zz/b6vF7hnHBSccTPj7t5WujNb/gc8rbPMc+8oPWVWW67Ufq95/ytjlCQTbwaUzuZXy0EAQyM5FIHboMKTGpz5z6ekBt/8ioyMgfOA6k5G2dbTBagErcT+sl7mbkF78foffeVe+vNTv/QvfBd+FUuos63fw5DLanX+cy19cQoHPSCJPf8YpzeWyagTR8eAsgSPZDV0ipRoNDQSB7FoMbftDbNMan3pe/3bcM6V3he0ZaS3Ieuxcuk+4DpsYptl/rFXR9h0uDvpYh0/HwzuZVh/DE59t4vutB/ly/QHiOIGdCPlmnLcFig9DH/ecRm0eUspLA0F5jhLYk1lh/kBNXDuqS4VtnqRyBQmdWO7qzoX276huwcv8wpIK26ppffLj8MmBdOd7a9iZV4hniWaXy8lHMffxecwd9JadLN7eOProQ8YzHHjgpdbPPA0ESnloIChv30pwFJ9UILDZ/J/WLRNjmdinDWCtWfC+80x62rLpKzsDF+FIkTVSKLPiSKHXF+3kgY/XsedwUbWpqx1O//33fLDWW7ZPPniTnrZs2kkeH8bcz7HvX2g0CY+KSpxBLwUatN1LoUkydB5lNQ/lN45Ryko1BhoIytu1yPpZw47iyvz32jNYds942jSLA2BCn9bMdQ7jhIly1woqGv7o1wC8WEl/wKs/ZnHxvxbR5S7/ZHeb9h/ze1++Y/n7rQexu6sUF8uX5JlExp34K0tcvZiw/TGYM7NRTLbqff//uOeDNXV70d2LoeMZYLNBcjrkba3b6ysVxjQQlLdrsfWgaHpyiVA/vGkk95/Xh1HdU/22t0qMY9kjF0GPSUy1/1BlG/3BgopNQx57DhdV2DbpyW/50r0uAvg3DXks3pFHSw4x3vYTc5yj2U8KV5XeyZ9LL8Gs/wieHw17VwbzK4aEp5Yzu5J5E7VSmGc9+Du516VO6apNQ0r50EDgy+WyagSda98s5DGwY3NmBugrACuzaeygS2kpRznTthqA6YM78NQlAyu93m/P7s7qByZWe9/rXs8kr8DKNlq+aQiseQcX2RcSLU5mO88CwGDjOedUbo592BpR89IEWNIwTUV13SIElM0f6Oiu5SWnW3mknJqDSSnQQODv4GYoOnRS/QNB6z6RQ6YpF9q/B6BXm0TO7F55Qr3fTehBUlw00wd3qPbSwx61Zi6XBpiuLLiYYf+GH5192GHa+u2bezgNfvUddB0H838P71wBjvpNYR2KVBvsXgy2aGjnDrQp6eByWMFAKaWBwI+3f6AeAkFUDMU9L2CiLZNEjtOmWRwt4qOrPe1PP+tX7TGlTkParHl8vu5AhX2jbWvoaMvlTefZAKz6w0Tm3jzKu78ktgXMmA1n3QsbPmHt1/W7aFxIKiG7l1pBILqJ9T7FSg2iHcZKWTQQ+Nq12FqIJrlrvdyu7ehriJNSJtuXcm6/ttXOTAaItgc/ftR30tiI9BRundCDGfavyTOJfO7KAKBZk2i6tkzwHvf3LzdbHaoj/4/jJpZl386vcN1QqvMageME7PnJ6ij2SHavJ6EdxkoBIc41FHZ2/WjVBmoyWP9ktB9MXmxHbk9e4Q0COx6dwpId+cRF27ng2R8qnBJMsPDVPD6aCwa255eju9Ledhjnd8v5t2MKnVq18C6w0yS6LBfRcwu28dyCbWR0bsH/ubox1LaREw4nsVH1k6+ozmsE+1aD84R/IEhIhdgk7TBWyk1rBB5H9sDhXfXTLOQhQsqIq2idvwwObnVvEoZ1TaFbq5rPag4kymbjgal9ad+8Caz4L3ZcnPmL2/jy1jHe/gYRoWm5DKWZOw+xzNWL3rKLuUs21klZgmGqmWRXY7vdE8l8A4GIVevT2cVKARoIyuyueiGakBl0hfXtdO7/+eWi9v2WHkiazwpp143qwitXDwk4qsg76czlhJ9ehy6j6dtvUIXjPvvd6Arblpme2MSQfKj+lro82VFD23IL/JYKZddiaJEGia39D0zppjUCpdw0EHjs9CxE079+75vYBib9EbK+g+WveDfbbcLK+ycEPGXJ3Wfz/o0jve/vPa8P43q1IikumkuG+K+c5m1z3/Y1HNkFg68JeM32zZtU2LbC1Y1SY6dg87c1/a1q7WT7CM7+60LGPLHAemOM1VHcMUBwT0mHI7vrfVSUUo2RBgKPI7uh45BKF6IJqdOvgK5j4Yv74XDZRKrm8TEBD2+dZI0wmj64A29f7/+QK5+awfs28xWrI7zXeUEXqyHRjrsAACAASURBVIg41pk0Wh1ewTvLdpM2a543LXao1GkfwaEdUJgDHYdW3JecbqUaP5RVhzdUKjxpIPC49G1r2GRDEIHzn7aegp/cEtTTUET4y0UDOKNrit/28k0rGZ1bwNG9sPl/MPAyiAocXHy9dFWG9/UyV08GyjbufW85ALnHQvsNurr8STWy253CO1BzX4pn5JA2DymlgcBXdMXmkXrTojNMeNBqwln5Rq0vc9mwTt7X90zpzdMzTocV/wXjhEFXVnv+wI7NObt3WXv6MldPYqWUfmKNuQ/1ZOM6nVm8a7HV/9KyV8V9niHC2mGslAaCRiXjWug0Av53NxzdB8CcXw9nwe1jg77EoE4t+NVo6yE3qnsqCdECy1+zmp4834IrsemRycz5tf+oqTPGWktxDrVtAkI089dHndcIOgwBW4CO9/hkKxup1giUCm0gEJHJIrJJRLaKyKwqjvu5iBgRyajsmIhgs8G0Z6xx73N/B8aQkZZMWmpC9ef6+P2knrx/4wh6t02CrV/C0exKO4l9xUbZibL7/y+R0Lw1W1ztGWKzhpDuPWIlu1udfZj5a/bVqFzBqLMaQdFhyFlf9SiwFM1CqhSEMBCIiB14FjgH6APMEJE+AY5LBH4LLAlVWcJKSjqcdR9sng9r36vVJaLsNgZ1amG9Wf6q1Uncc0rtitM0lmWuHmTYNiO4uPTfS3g3czdTn/mBG974CafL4HC6SJs1j/8syqrVPXzVWY0gOxMwgTuKPZLTNc2EUoS2RjAU2GqM2W6MKQFmA9MCHPcw8Gcg+DUYT3XDboD2GfDp76Egt/bXObLH6iQ+/fKgOol9tU6KBSClaQzLXL1IkuP0FGud39/PWe09Lv3uT+l2j5WG4tH5Jz/xrM5qBLuXgNisz7EyKelwdA+UHK+jm4afrzceYM5yXb850oUyELQHfJPKZ7u3eYnIIKCjMWZeVRcSketFJFNEMnNzT+LBGC5sdpj2LJQUWFlAa2vNu9YQyUFX1fjUNs2sjvOkuGiWGauz1dM8VBlbFekvtuYU8LcvNlf7jb/OZhbvXgytT6t63WlPh/GhHXVzzzA089VMbn+3/iYMqsapwTqLRcQG/A24rbpjjTEvGGMyjDEZLVtWnqr5lNKqF4y5E9Z9AOs/rt01sr6zRswkB14XoSovXDGYh6b1Jb1lAtkmlX0mmSHuDuPKiFhLcRaVOL1rIhwtLuVocSlXvLSEp7/aEnAdZl91USOw44Ts5dXPEvdkIdUOYxXhQjl7ag/gO821g3ubRyJwGrDAnUitDfCxiEw1xmSGsFzhY+RvYd2H8NVD0Gdqzc51Oqzhk/1/Uatbt06K48rhaQCcc1pblm3q6Q4EBgj8zf9YsYPu9wTOVto8iBTbAK46iAS9ZBeUFvrnFwokRbOQKgWhrREsA7qLSBcRiQEuAbxfbY0xR4wxqcaYNGNMGrAY0CDgyx4NAy+FvC1+M46Dsm+V1bSUNqr6Y6vx3OWDie92Jm0ln9+fEdxci+YcQyjLneRpEfphWx4/bjt40mWqSoZts/WiukAQmwgJrXQugYp4IQsExhgH8BvgM2AD8I4xZp2IPCQiNfx6G8G6jrF+7lhYs/OyvrN+dh5Z9XFBGj/J6uef2aH6IaPtyeXH2Ft4NvppbO5gcKSoFIBb3lrBpf9ewqb9x0ibNY9b315J1sFCb9+B7zyF3fm168QdbNsMSe2hecfqD05JhzwdOaQiW0j7CIwxnxpjehhj0o0xf3Rvu98YU6HR2xgzVmsDAbTqYw3/3F7DQLDzB0jtUTHr5smUI64ZcfuW+m0+rX1ShUNvjvqAOEqYYl/K3VGBZ0lPetJKZPf+ij2M/csC3liyC/DvI7jm1WXe17e9s4qXvg+uU3ewbXPVw0Z9paRrjUBFPJ1Z3NiJWLOCty8IPr+Dp3+gjmoDgDXZreMwZNciMu8d79089+Yz/Q7rLPuZbv+W15wTecUxieui5nO1/X/VXv7z9Qe4/d1VFBSXraq2NaeAD1ZkszXnGO/9lM3Dc9dXe5225NFe8vjntlQeCeJ4ktOh4ACcOFb9sUqdonSFsnDQZYw1FDRnA7SuMCevov2r4cTROukf8NNpGGz5jFSp/KH526j3KSWKfzqmkUcS7SSP+6P+w16TwueuIZWe9+1ma1iwrVw/9O/ertnQxsHu/oH5Rzqz5vsd3Htexc/L4XSVzaD2TT7nWdxeqQijNYJw0HWs9XP7guCO3+le4rKuA0HnEdbPXYv9NnumD9w9RLjA9gOvOSeSS3Nc2Pht6U2sMuk8Ff0sA6X60Tnfbq59R7LjyH5mRs2n0MSywXQKeMyjn26g2z3zcTjdHdme9Yu1eUhFMA0E4aB5R+uBFWyHcdb31vGJbeq2HO1OB3ss7Frkt3nVHyby9IzTmX7sPxwnlucd1poHlwzpyIZHf8Z1JbeRY5rzYsxf6CQHqrzF/qPVTzBftfswW3MK/LZlfv8ZuX8bTh/ZyazSX+JwV3Y9ndRgDU19/lurY3ieJ0+SZ1KZdhirCKaBIFx0HWM94J2lVR/nclqrrdV1bQAgKhY6ZMDOH/02J8VFM7XVQZKzPuVl52TatrXWQm6VGIuIkEczri69k2gxzEt+iubUvj3+sfkbmfbsD4z/mzsoGgPLXmTgV5dSYqK4sORBPnGN8B4/4MHPeSfTGnr75YayIPTb2SsB2HLIiaNp26BqBBc8+wMvB9lhrVQ40UAQLrqOteYF7Fle9XH718CJI6EJBGD1E+xbRXz51FDf/AnimnH9nX/n49+M5M7JvbhxnDVz91djulLavCvNZs4hsXgf/475K7GU8Bv3/pr418KyB3bPWR+Q85+ZMO82tiRkcH7JI6w3aRXOeX7hNrbmHGPpjny/7R+syGbC379l6ZEWQU0qW7n7MA/NXc/fv9hc43Ir1ZhpIAgXaWcCUv0wUk//QF2OGPLVaQQYJwNtW2nWxD1bOHu5lS11+M3EJSYTZbdxw9h04qKtdQDuOqc33995lhVELnyeIbbN/CX6X9w+sQf/unwwL16ZUaGTuDodJIf3Y/5A6rYPmJ1wOf9q9yeOEjiv0LbcQsb/7VteLPdt3tMRnWXa1CjNxFNfbalZYZVq5DQQhIv4ZGg7oPoO46wfoEUXaNa+6uNqq+NQEBsvjy3lh1lnWdu+ecRa5GXYr6s/v+/P2D3g/zjfvhgOrGPyaW0Y36d1hXUQwFpXIZCxtpXMjbmHDpLLzNLbmZU3hY9W76/1r7TDtIGifCg6BFj5kt5ZtrtO0l00tLyCE9z69kqOlziqP1hFLA0E4aTrWMheCicKAu93uawaQaiahQDikqB1X+L2LaFpbJTVX7Dtaxj1OytlQxA6nv0r68W2r73b+rVvBsCGhybz14sG8OKVGdw4tuKKat0km5ein2CvSeW8kj+ywHX6Sf9KWcbqVL/l2fd48bvtvPDtdu54bzXv/VSWnrl8UHC6DDvzCms9+7kmDhWWsHznoVqd+7cvNvP+ij28p6mmVRU0EISTrmPB5agwascrZx0UHw5tIACreSg70+q4/vqP0LQ1DLku+POT2kHL3n6B4KWrMvjvtWfQJMbOzwd3YHyf1ogIU/r5j3yaaMvELoarSu5kt6mbWdPbTVsAJH8bj8zbQF6BlSF1yY58nO4A4Cw3ma+o1MmYJxZw5uPfnPT9j5c4OPfp71idfTjg/hn/XszPn/sx4L6gVZEiXCkNBOGk0zBr+GZlzUNZ31s/Q9U/4NF5OJQehx+fhp3fw5m3QUx8za6RfpZVmyi1lr5sHh/DqO6pFQ77x4xBbHx4MrOvt1JKj7OvZJWrK7k0P+lfw2O3aYXTCF1tVvPS3NV7AZizPJunvrQ6hp3lagTHT/g3tazJPsKwP33FoWrSbAeyavcR1u09yiPzNgTcv3G/NcqqTtdzbuSOFZeSe+xEQxcjYmggCCfRTaDTGVUHguadg0u2djI6uRe4//oRK7lbLRa+odtZ1trMns7tSthtQly0nWFdU2hGAYNkCwtcA5l7c1mtZ+bImq+34KuEaPaaVNLECgQ5Pg+g5busJhlHuUCw+UBZ89zOvEKe/WYr+48W8+O2vFqXo7rv7OWD0ans7L8uZMgfv2zoYlSwLbeABZtyGroYdU4DQbjpOhYOrK24hKXLZX3DDnWzEFgT1Vp0sVY/G/17iI6r+TU6jbBqN9uCb1r57PxS7GL4xjmQ09x9CgC921p9E57Wj8l927Dk7rOrvNbzVwz2e7/DtPEGAl8/bLUe7CdKnX7bL3+pbIntMU8s4PP11rkOl4uaCnZVtvLB6FSW00hrA2f/dSFXv7Ks+gPDjAaCcNN1rPWz/Czj3A3WyJf6CAQAPSZDak9rPeTaiIm3mpi2fhX0KW0OfEeeSWRHTA+/7dMHd+C/157BP2ZYHcci1sI6HVpUvnbCpL7+fQ87TBu6yH4I8FCe/tyPDH6k6m+nnmd0rb61u0+prhnfFUFNQ6p+aSAIN20HQlyzis1DWSGeP1De5Efhhh+sxXNqK/1sK4Ad3Vv9sS4XbP2CE53H8ckt1hoNb153Bq9cMwQRYVT3VO+ayZ4H5oc3jeTqEWkAtPBZIe29G0ZQXpZpQ5IcJznArOfMGozYufWdVRSecJB1sLBCGoyVuw6xfncuFB+BYwfgUBbkbiIhby2DZDNxrqrTa0RSjUDVL80+Gm5sdmty2faFVnoFz9fIrO+gWSdo0bl+yiFyckEArA7jL+6zmodOv6zqY/eugON5tBsyDVKsjukR3fw7l8/snsqgTs25faI1/yC1aSz3n9eHXm0SuXBQBwpOOIiPsXsnuvna4R5C2kX2kW8qrrFQExv3H/OO8sl67Fxr49J/03fenUSLs8LxA4D3YyEvJxU2PQM9zwl43VNhXoNqnLRGEI66joUju+CQe6asMe75A/VUG6grrftaQ0+3BdE8tOVzEJsVPCqRGBfN+zeOpHvrsvkMNptwydBOxETZSE6ICRgEHpza1zuXoIut6olpD19wGpsfCfyg9lgTaBjoT6+xy7TiidKLYeIfKZr4ON/3fQjz85fZMOZ5fl3yfxTaEuGtS3C8fRWlRyqWozbNTho6VDA0EISjrmOtn57modyNcDyv/voH6oqI9WDf9o3V9FOVLZ9DhyHWDOs68uYvz+Cmcemc178t2aYlDmPz6zAekZ5S4ZztuQXERFX9Z/PAJ2UL4uw5XMQz738J+9cw2zmOZ50XsLPnNdyTfQaXL+/Govgx5Hccz/9cQ7mn1T/grHtxrp9L8ZMZsPJNv8WIImnUULhauiOf/FoMIW5oGgjCUUo3a9imJxDU1/yBUEg/y+rk3l/FAjQFObD3J+g+oU5vPSI9ld9P6kVK01hWPXguu0wr0mQ/H940kheuGMxfLx5Q4Zwrh6cB8MgFp9GrTSJNAtQwfI187GsOLf8AgM/cC/OMeWKBd0ZycanT+6x32aL5sf01TCl5lA3OdvDhDex6ejIdxBqueODoCaY9+wP7jhQFvpnTAXnb2PL9ezi+fxo++S3Xbf0N38feQt9vriNvwT/h8K6Ap27LtfozvtmUw0cr91T5O6nKXfz8Ii5+vpIJn41YSPsIRGQy8BRgB140xjxWbv+twHWAA8gFZhpjdoayTKcEz/KVmz4tSyuR1AFapDVwwWqh6zjr59avrPUOAvGMLOo+MWTFSIiNIsu0oavsp0/Hsslq390xjqU78rntXStQdXSPRLp8WGcuH9aZO+as4p3MqtM3TLRnssHVkV0+M6FLHFYNaOarmfx8kJW2WxAu/fcSoD2/KLmPy+xfcWf+bD6PWcFzjvPZOv8nzti7h+1vv0ubjrG8t2gDg9tE0yXRZXW4528HZwnd3ffIN00psHUgy9WdgUVbSVlwFyy4i5IWPYjpPRm6TyQKBw6iOPuvC8l67FyucQ+NHNujFXa7WGlEVFA8NbbygwSC5VksKVDerVAL2X9lEbEDzwITgGxgmYh8bIzxXUh2BZBhjDkuIjcAjwO/CFWZTildxsDKN6xv0lnfW9+swzGNQNOW0Ka/1Tw0+vbAx2z5HJq2sY4LoaT2PUnb96FfJ3zH5Hg6JsfTq20iP2w9WOGP9JEL+pHesimPzt9Iz9aJbDrgP+oomaNkyCaecV7gt31V9hHva09Oo++3lq3OZrDxX+cEvnIO4uHol7kteg7sgZ9Fg2uvULQ/nhESS8H+JhS4WlIc14GUMybxyqYo5u5NZLtpy2F8cz8Zuso+ftlmCx0Pfs/Ixc8hPz7NT7FNWOgawHvO0eCa7D36yleWsib7MK/PPIO2zeNIb2lldt13pIimsVFE221sOVBA99ZNOXy8lDbNajCXpDAPFj0Dy1+xarcDZsBpF0KTFtWfW1IIm/8H6z+GwoMQk2ANRY5OKPc6HmxRVr+SzQ5i93+d2NqayxIVE3y5fRhjkHJ/b6XOms8h8XX6w1+QFBddlszRx74jRdz4xk/cOqEHZ3ZveVL3CSSU4X4osNUYsx1ARGYD0wBvIDDG+M4mWgzUclB6BOpqDaFk6YtQmBuezUIe6WfBometBeTLJ65zOqzO5N7nhzzQZQwaAp++Des/gr7+D+6+7ZrRt12zCufERNk4u3drHp2/kX4dmlUIBOPty7GL4XNn5es1V2UfKVxXejvtSvMoxU4BTSgiFr95yO4KyUN9+/LQnnWVXEnYbtpx1752wBhaSgnLZkQz940XmWTP5Hz7YoqfeI3bo4bxrnMMq6y1fLwT5zyjn4Y/+jWdkuMZnpbE2hWL6Sz7WeVK57s/XYWtulzihQc5sfBJYla8hJQWWaOj8nfAvFvhf3dBrykw4NKKAwJKi6wvA2vfh82fgaMIV0IrJCUdKdgPJcetlCclBdZrVzWLN3nEJnGs41i2J49mwLiLoElzit0TBwMNKvDlchnsx3OsprbDO+HwLpw5O3gteqX132fhOuuLS9v+kNiWEqdhwaYcJvatuGpgfmEJyQkxHCt2cKw4cJbYnKMnWLHrMCdKTy7YVCaUgaA9sNvnfTZwRhXHXwvMD7RDRK4Hrgfo1CnwWrQRJ7GNlbht1VvW+3DrKPaVfhb88KRVsyk/dDJ7mTXuPoTNQl6n/dzqoH33Kth/O4y72/oGWY1urZry8W9Gkt6yKXPcWT7P7J7KviPFTDqUSbZJZZ05mWG9wl4q5mEq7/6PKgsCFeWWxLAkdhh3O4Q/OK7hLNtPXOxcyA32j/lN1EcscfXiHcdYPnUNpZgYPvziGwp3LOUPUcsZULCNvut3ERtb1inqeOYfOLqcxc4Ww2h/+gTiE3yG4BbkwA9PQebLRJUU8ZFrOBfc/Hdo1YtPV+9l6Y9fc1/HVdjXzoF1H0DT1twdlcEaVxfMnPeRTfOhtBDiU2HgpWxMHc+UD538Ij2NRy/s573Ns99s5WhxKXdNSLcCg8sJxlnxpzGQuwk2fUrxTx8xQD6GzFnQeSSPbe7Ml65BfHfXJOTYPji6D47ts5reju3jv9GraSv52B6dCQ7/eR/FJpEWkkpHcuCbP5btiE9lT3Q62/JasXH0OHp1auutnazde4zHPtvM7ZN6Mdy2mSITa92raWu///fyj1ufdYuE2tVgqiOhSmQlItOBycaY69zvrwDOMMb8JsCxlwO/AcYYY6qcW56RkWEyMzNDUeTwM/9OWPIvSGwLt24Iz6YhAMcJ+HOaNUt5yhP++7580Epud8d2ayJdqJUWw6e3wYr/QrcJ8PN/B9dk4bY6+zBpqQkkxUVz/Ngh7H/pzhvOs3nIcSWPT+/Pgx+vo7Ck4lyCxqI1+Vxo/56L7AvoattPgYnDhZAkVgf1cRPLGtOF1a6urHKls8u0IsO2mUlx6xjgWEOclHLCRBOTPhJJP5u92Ttosf6/xIqDkt4XMmXlMLabdmx+5Bz+tXAbf3Ov9vaH8/vw53lr+GZqMW13fEjpxv8RLU4csS2I6jsVTruQ0o4juO/jjcxeVvb98v0bRzCok/XfJ23WPMBn7kYAzy/cxrCuKQxw9wN1nfUJA2Ur7407TNG6ucQfqWSlOrHjbNqa1Ufi2WeSOXPIIJ5YUsTlk86kR8++9P37egopm8meQBG9ZSdvT03AfmA1ezYsplVxVsB5JAHZoqy/66T20Kw9m4qSeHOji19dNoN2fYYHd43yv4LIcmNMRqB9oawR7AF8s591cG/zIyLjgXsIIgiocrqOtQJB2qjwDQJgrYWcNsovLbXXli+sJHf1EQTAyps09RloN8gKtC+Mg0vesOY8BKF/h7KO5iY7FyBSymfuZqGLMzpytKi00iyjjcEBknnOOZXnnOeTIZu40P49LoRVJp1VrnS2mva4yg02XO1M5+XCc4ilhKG2jYy2rWbs1lV0376AVsbGh65RPOOYRtaKtt5zetzrX/n/emMOxS47t6zswLKsy0jmfNJkP6uLuzK9tAuPde3Pqqx8vyAA8OSXW5jQuxWXDyurcb2zbDd3vLeaP/+8HyPSUznz8W/o0zaJu6b04tH5GwGY0q8NN47thgsbP5kerOw1gp99PZQ02ccY22qc2LAltaVPz15cNG4o9sRWLNh0kGtfs76EDtufzGJnPq9/Clmj+1CI/+p3hTQh0/SiYMBEmsVH82TpKj5avoO/nJXAC99sxIahb9umbNp3BMFw8eB2vL88mwQp4qZBcbQmj+iCvbQyB5E9y+l2eA8PRpdSvDsZahkIqhLKQLAM6C4iXbACwCXApb4HiMjpwPNYNYdTL6VfqKWNsmYT95nW0CU5eelnWe3Ah3aWzY4+uhcOrIEJD9VvWURgyLXQph+8fQW8OB6m/gP6Ta/ZZTbOhfgUMovLciNdNLgj6/ce5Z5zewfMXxRlEy7K6MjMkWlM+Pu3ANx1TtkDDOC1mUO56uWlAPRtl8S6vUdr81tWV3oyTS8yHb2CPuMEMXzn6s93rv78kctpQx4G4QDVz/1Ici97uizLSueRT5J3hvfsZbsZnp7CM19X/Lb+7eZcvt2cS7dWZX1Ld7y3GoA731vDwxecBsD6fUe54qWl3mM+XbOfT9eUzRn52T/dM8FNW7Kc7oB1CFgMec0K2Xt4Pb3alN1j8Xb/9a8rM+Chz3n318Ox24QSornl6xKgKwBWtnNrJFny8dYsMYlg4Kf10Rw+XtbP0bttEhuLDtPKdozFo0LTRBqypiEAEZkCPIk1fPRlY8wfReQhINMY87GIfAn0A/a5T9lljJla1TW1aegUlbsJnh0K5z0JGddY25a/Bp/cAjcuhla9G6Zcx/bDO1fB7sUw/Dcw/kGwB/H9yVECT6RD76ksG/gwh4+XMqGP/0I6abPmcekZnbAJLN95mA37jnJxRgcen27NX1i0LY/4GDsDOjZn+c58Vu4+wshuKfRqk+RtBtnx6BS63PWp95p3T+nFnz61gsbSu89m6J++ol/7Znxy8yiGP/oV+45Unc9I1dzEPq35fP2BerlXVc1e1WmopiGMMZ8Cn5bbdr/P6/GhvL8KI6k9rLkQ274uCwRbPodmHaFl8N9K61xiG7jqE/jsbmvI49G9cNEr1Z+X9S2cOAq9z2NIWuBvxDseneIdguhwunj6qy1cO6qrd/9wn5nNgzsnM7hz2XU2PDQZERARznQv6PPdloP8IqMTQ9KSKXG4aJUUxyMXnMb43lYAquo7X4zdxvs3juCT1XtJTYjl9cVZ7M63+gWi7UKp0//kf142iN+9vZITjupHsQzu3KLWS22Gg/oKAqGks0VU4yAC6eOs8eFOh7XWwfYF0P/ihu//iIqBc/8CTVtZo0FO+zn0Pq/qczbMtcazeybMBeA7Dj3KbuNWd7K8YDSJKRtR8p9r/Qfjnd6prHPbt+38gal9uO+jdfzzskE8t2AbI9JT6N+hORc/v4ivbhtDx+R47zoPvxzdlXcyd3PHnNX0bpvEC1dk8K+F23j1xywuH9aJ8b1bV/jPsvbBScTYbSzanudtuvrujnF0TI7n2825XPnyUupKoOB0qvv9pOD//6ipkDYNhYI2DZ3C1r4Pc66Ba7+wxo6/PhVmzK40G2e9c5bCC2PheD7ctATiKslS6nLB33pZS4te/Hq9FrEurd1zhPP+8T192yUx75YzK+zvfd//KCp1MrFPax6Y2pd2zStf/wHgyPFS/rtkJyccVu0H4OoRabz3U7bf+PnkhBiOFJXidBkGdGzO7vzjfvl7Pv7NSPq1b8aVLy/luy1lk/AW33U2wx4tS2B4/eiudGzRhPt8htWe1asVX2+svjuyW6um3hnCT10ykLE9WjHgoc+9+1++OoOnvtziNzGwvF+N7krOsRNM7NOa+Ngonvl6C8uyDpHeMoFtuYUBzzmjSzJLdlTsf3jswn5cMvTkhs43WNOQUjXSdSwgVvPQiWNgj4Euoxu4UD7s0XD+0/Di2dYynVMeD3zcnkwoOAC9qqk1NHJ29wSxypLdeWoEj0/vT/P46se3N4uP5qZx3TDGsGjbQZZlHSLaLnxz+1gy3B3n790wgm6tmpIUF+WtMS3LyueReRt44YrBNGsS7Z3sVb4mBLDtT1N4/tttXDk8zZse45x+bck+VIQAfdolUeJw4TKGaLuNnz/3o7ez/f0bR/DTzkM8Mm8DQ9KSuffc3nRJTaBzSgIAqx+YyJzMbK4ZmYaIMCQtmdlLdzP5tDZ0TLZSozucLs7+20IyOicz65xefrW+5xZYnd0PTzuNS1+0Juq9cs0Qb1oPgNnXD2P83xYyslsqbyzZxTmntSE+xs7PB3eo9vM9GRoIVOMRnwztB1m5hYqPWKOiYhIaulT+OgyGob+EpS9A/19Y78vb8Ik1Drw+JsGFULTdeojZK5kx7FkIqNoZxeWICGN6tHQHAhupTWPZ9Mhkducf9xv94zEkLZmPbgpu5rzdJtw4tpvfttSmsaQ2jfW+j/ZJEzLvljMpOOFgTfYRBnVq4f2dRnVLZWzPVn7XSYqLZuaosvWxE+OizZzSWQAACc5JREFU+eXorn7HRNltLPx94OZATzwVEf4x43S6tkygb7tmvD5zKPuPFjOlX1tEhK9uGwvAQ9NOC+p3rgsaCFTjkn4WfPsXwJR1Gjc2Z91nPew/+S1cv8B/FJExsHGuVZNp0ryyK4SF9JZNuWlcOr/ICNwk8co1Q3jtxyyaxtT8MeJpRuqSagX62Ch7wCBQH5rGRnk75gd2bM6q+62x/3Xt2lFdWLojn55tEhmeUDYQYHSPus8dVFPaR6Aal52L4BV38rObf4KU9IYtT2XWfwzvXAETHoaRt5Rtz9kA/xwG5/4VhlzXcOVr5KzmoTyGp6dUSN6mQqOqPgJdj0A1Lh0yICYRkrs23iAAVhK8nlNgwaPWJDiPDXOtnz1rP947EogII7qlahBoJDQQqMbFHg0THrSaXxozESsvkthg3m1lg/Q3zrVWUktqW/X5SjUiGghU4zPkWis/fWPXrAOcdS9s/cLKmnl4N+xbCb20NqDCiwYCpU7G0Ouh7UD43ywrhTVAr/MbtkxK1ZAGAqVOhs0O5z9lLQ604FErHUZqt+rPU6oR0UCg1MlqNxCG3QgYbRZSYUnnEShVF8beZeVHyri2oUuiVI1pIFCqLsQ2hcmPNnQplKoVbRpSSqkIp4FAKaUinAYCpZSKcBoIlFIqwmkgUEqpCKeBQCmlIpwGAqWUinAaCJRSKsKF3cI0IpIL7Kz2wMBSgYPVHlX/Gmu5oPGWTctVM1qumjkVy9XZGBNwObSwCwQnQ0QyK1uhpyE11nJB4y2blqtmtFw1E2nl0qYhpZSKcBoIlFIqwkVaIHihoQtQicZaLmi8ZdNy1YyWq2YiqlwR1UeglFKqokirESillCpHA4FSSkW4iAkEIjJZRDaJyFYRmVXP9+4oIt+IyHoRWSciv3VvTxaRL0Rki/tnC/d2EZGn3WVdLSKDQlw+u4isEJG57vddRGSJ+/5vi0iMe3us+/1W9/60EJapuYjMEZGNIrJBRIY3hs9LRH7n/m+4VkTeEpG4hvi8RORlEckRkbU+22r8+YjIVe7jt4jIVSEq1xPu/46rReQDEWnus+8ud7k2icgkn+11+vcaqFw++24TESMiqe73Dfp5ubff7P7M1onI4z7bQ/N5GWNO+X+AHdgGdAVigFVAn3q8f1tgkPt1IrAZ6AM8Dsxyb58F/Nn9egowHxBgGLAkxOW7FXgTmOt+/w5wifv1v4Ab3K9vBP7lfn0J8HYIy/QacJ37dQzQvKE/L6A9sANo4vM5Xd0QnxcwGhgErPXZVqPPB0gGtrt/tnC/bhGCck0Eotyv/+xTrj7uv8VYoIv7b9Qeir/XQOVyb+8IfIY1STW1kXxe44AvgVj3+1ah/rxC8kfc2P4Bw4HPfN7fBdzVgOX5CJgAbALaure1BTa5Xz8PzPA53ntcCMrSAfgKOAuY6/6f/6DPH673s3P/wQx3v45yHychKFMzrAeulNveoJ8XViDY7X4QRLk/r0kN9XkBaeUeIDX6fIAZwPM+2/2Oq6tyldv3M+AN92u/v0PP5xWqv9dA5QLmAAOALMoCQYN+XlhfLMYHOC5kn1ekNA15/oA9st3b6p27eeB0YAnQ2hizz71rP9Da/bo+y/skcAfgcr9PAQ4bYxwB7u0tl3v/Effxda0LkAu84m6yelFEEmjgz8sYswf4C7AL2If1+y+n4T8vj5p+Pg3xdzET69t2g5dLRKYBe4wxq8rtaujPqwdwprs5caGIDAl1uSIlEDQKItIUeA/4P2PMUd99xgrl9TqWV0TOA3KMMcvr875BiMKqLj9njDkdKMRq6vBqoM+rBTANK1C1AxKAyfVZhmA1xOdTHRG5B3AAbzSCssQDdwP3N3RZAojCqnUOA34PvCMiEsobRkog2IPVFujRwb2t3ohINFYQeMMY87578wERaeve3xbIcW+vr/KOBKaKSBYwG6t56CmguYhEBbi3t1zu/c2AvBCUKxvINsYscb+fgxUYGvrzGg/sMMbkGmNKgfexPsOG/rw8avr51NvfhYhcDZwHXOYOUg1drnSsgL7K/f///7d3N6FxVWEYx/8PQrVuRGvFRRdjIVWoYBWFIl0ElaAiIlKoWFCs4AdoVyLVrtx15U4QQRC0dCFqyMoKarVWyqSGtCmtrQEFi0QQpSgFKfF18Z5LbqaJJemdGeQ+PxiSuffmzpmTO/PO+Zj3bACmJN085HJBXv8fR+qSrfUb+1mutgSCSWCkzO5YQw7cTQzqwUs0fxc4HRFv1nZNANXMg6fJsYNq+1Nl9sJW4Hytyd+YiHgtIjZERIesky8iYifwJbB9mXJV5d1ejm/8U2dEzAE/S7q1bLofOMWQ64vsEtoq6dryP63KNdT6qllp/RwExiRdX1o7Y2VboyQ9SHY/PhoRF3rK+4RydtUtwAjQZQCv14iYiYibIqJTrv9z5ISOOYZcX8A4OWCMpE3kAPBv9LO+rnSg4/9yI2cCnCVH1/cO+LG3kc30E8B0uT1M9hd/DvxAzhK4oRwv4K1S1hng7gGUcZSFWUMbywU2C3zIwuyFa8r92bJ/Yx/LswU4VupsnJylMfT6At4AvgdOAu+TMzgGXl/AAXKc4iL5JvbsauqH7LOfLbdn+lSuWbIPu7r2364dv7eU6wzwUG17o6/XpcrVs/8nFgaLh11fa4APyjU2BdzX7/pyigkzs5ZrS9eQmZktw4HAzKzlHAjMzFrOgcDMrOUcCMzMWs6BwFpL0l/lZ0fSkw2f+/We+982eX6zJjkQmGXSrxUFgto3iZezKBBExL0rLJPZwDgQmME+MsnXtHK9gauUOfQnSz765wEkjUo6LGmC/EYxksYlfVfyxj9Xtu0D1pbz7S/bqtaHyrlPSpqRtKN27kNaWINhf7/zy5hVLvepxqwN9gCvRMQjAOUN/XxE3CPpauCIpM/KsXcBt0fEj+X+roj4XdJaYFLSRxGxR9JLEbFlicd6nPzW9B1k/phJSV+XfXcCm4FfgCNkHqNvmn+6Zou5RWB2qTEy18w0mS58HZnXBaBbCwIAuyUdB46Sib9G+G/bgAMRMR8RvwJfAVWa4W5EnIuIf8hUDJ1Gno3ZZbhFYHYpAS9HxKKEYpJGyZTY9fsPkIvPXJB0iMwvtFp/136fx69PGxC3CMzgT3IJ0cpB4MWSOhxJm8rCOL2uA/4oQeA2Mn985WL19z0OAzvKOMR6cqnCbiPPwmyV/InDLDOczpcunvfINRk6ZH56kaulPbbE330KvCDpNJkN8mht3zvACUlTkam9K5+QSwseJzPSvhoRcyWQmA2Fs4+ambWcu4bMzFrOgcDMrOUcCMzMWs6BwMys5RwIzMxazoHAzKzlHAjMzFruX4HK5hKjX77EAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_x = range(len(train_curve))\n",
    "train_y = train_curve\n",
    "\n",
    "train_iters = len(train_loader)\n",
    "valid_x = np.arange(1, len(valid_curve)+1) * train_iters*val_interval # 由于valid中记录的是epochloss，需要对记录点进行转换到iterations\n",
    "valid_y = valid_curve\n",
    "\n",
    "plt.plot(train_x, train_y, label='Train')\n",
    "plt.plot(valid_x, valid_y, label='Valid')\n",
    "\n",
    "plt.legend(loc='upper right')\n",
    "plt.ylabel('loss value')\n",
    "plt.xlabel('Iteration')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(net, \"./1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = torch.load(\"./1\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
